# coding=utf-8

import threading
from datetime import datetime
from time import sleep

from difflib import SequenceMatcher


class Info:

    def __init__(self, key, count):
        self.key = key
        self.count = count
        
        self.list = []

    def __str__(self):
        return "%s: %s" % (self.count, self.key)


def p(str1):
    if 0:
        print("%s" % str1)


def _wordcount(sub_list, str1):

    found = False
    for info in sub_list:
        ret = SequenceMatcher(None, info.key, str1).quick_ratio()
        if ret >= 0.8 :
            found = True
            info.count += 1
            return
        
    if not found:
        sub_list.append(Info(str1, 1))


flag1 = 0  # 目标是分段数量
flag2 = 0  # 目标是1
all_list = []


def do_str_match(lines, start, end):

    global flag1, all_list
    
    sub_list = []
    
    p("info: do_str_match(%s, %s) BEGIN" % (start, end))
    
    for i in range(start, end + 1):
#         p("info: do_str_match: %s" % i)
        _wordcount(sub_list, lines[i])

    p("info: do_str_match(%s, %s) DONE" % (start, end))
    
    all_list.append(sub_list)
    flag1 += 1


def do_combine_dict(sub_list1, sub_list2):
    
    global flag2, all_list
    
    for info1 in sub_list1:
        for info2 in sub_list2:
            ret = SequenceMatcher(None, info1.key, info2.key).quick_ratio()
            if ret > 0.8 :
                info1.count += info2.count
                sub_list2.remove(info2)
                break
    
#     for info in sub_list2:
#         dict1[key] = value
    sub_list1.extend(sub_list2)
    
    all_list.append(sub_list1)
    flag2 -= 1
    
    
if __name__ == '__main__':

    # 1.读文件
    with open('7000.txt', 'rb') as f:
        lines = f.readlines()
    
    # 2.分段
    nrows = len(lines)
    count = 20
    part = nrows // count
    flag2 = count
    
    # 3.分段统计
    threads = []
    for i in range(0, count):
        begin = i * part
        if i != count - 1:
            end = (i + 1) * part - 1
        else:
            end = nrows - 1
        
        t = threading.Thread(target=do_str_match, args=(lines, begin, end))
        threads.append(t)

    print(datetime.now())

    for t in threads:
        t.start()
        
    # 4.周期轮巡
    while True:
        sleep(5)
        
        # 退出条件：分段统计完成 & 字典合并完成
        print("info: flag1=%s flag2=%s qsize=%s" % (flag1, flag2, len(all_list)))
        if flag1 == count and flag2 == 1:
            break
        
        # 字典合并
        while len(all_list) > 1:
            sub_list1 = all_list.pop(0)
            sub_list2 = all_list.pop(0)
            if len(sub_list1) >= len(sub_list2):
                t = threading.Thread(target=do_combine_dict, args=(sub_list1, sub_list2))
            else:
                t = threading.Thread(target=do_combine_dict, args=(sub_list2, sub_list1))
            threads.append(t)
            t.start()
          
    for t in threads:
        t.join()

    print(datetime.now())
    
    # 5.排序
    result = sorted(all_list[0], key=lambda x:x.count, reverse=True)
    
    print(datetime.now())
    
#     print("%s" % result)
    
    # 6.校验总和
    sum1 = 0;
    for info in result:
        print(info)
        sum1 += info.count
    print("info: sum=%s" % sum1)
        
    print("*** OVER ***")

